{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearBottleNeck(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, t=6, class_num=10, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU6(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        self.residual = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, in_channels * t, 1),\n",
    "            nn.BatchNorm2d(in_channels * t),\n",
    "            f_activation,\n",
    "\n",
    "            nn.Conv2d(in_channels * t, in_channels * t, 3, stride=stride, padding=1, groups=in_channels * t),\n",
    "            nn.BatchNorm2d(in_channels * t),\n",
    "            f_activation,\n",
    "\n",
    "            nn.Conv2d(in_channels * t, out_channels, 1),\n",
    "            nn.BatchNorm2d(out_channels)\n",
    "        )\n",
    "\n",
    "        self.stride = stride\n",
    "        self.in_channels = in_channels\n",
    "        self.out_channels = out_channels\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        residual = self.residual(x)\n",
    "\n",
    "        if self.stride == 1 and self.in_channels == self.out_channels:\n",
    "            residual += x\n",
    "        \n",
    "        return residual\n",
    "\n",
    "class MobileNetV2(nn.Module):\n",
    "\n",
    "    def __init__(self, class_num=10, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU6(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        self.pre = nn.Sequential(\n",
    "            nn.Conv2d(3, 32, 1, padding=1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.stage1 = LinearBottleNeck(32, 16, 1, 1, activation = activation)\n",
    "        self.stage2 = self._make_stage(2, 16, 24, 2, 6, activation = activation)\n",
    "        self.stage3 = self._make_stage(3, 24, 32, 2, 6, activation = activation)\n",
    "        self.stage4 = self._make_stage(4, 32, 64, 2, 6, activation = activation)\n",
    "        self.stage5 = self._make_stage(3, 64, 96, 1, 6, activation = activation)\n",
    "        self.stage6 = self._make_stage(3, 96, 160, 1, 6, activation = activation)\n",
    "        self.stage7 = LinearBottleNeck(160, 320, 1, 6, activation = activation)\n",
    "\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(320, 1280, 1),\n",
    "            nn.BatchNorm2d(1280),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.conv2 = nn.Conv2d(1280, class_num, 1)\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x = self.pre(x)\n",
    "        x = self.stage1(x)\n",
    "        x = self.stage2(x)\n",
    "        x = self.stage3(x)\n",
    "        x = self.stage4(x)\n",
    "        x = self.stage5(x)\n",
    "        x = self.stage6(x)\n",
    "        x = self.stage7(x)\n",
    "        x = self.conv1(x)\n",
    "        x = F.adaptive_avg_pool2d(x, 1)\n",
    "        x = self.conv2(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        return x\n",
    "    \n",
    "    def _make_stage(self, repeat, in_channels, out_channels, stride, t, activation = 'relu'):\n",
    "\n",
    "        layers = []\n",
    "        layers.append(LinearBottleNeck(in_channels, out_channels, stride, t, activation = activation))\n",
    "        \n",
    "        while repeat - 1:\n",
    "            layers.append(LinearBottleNeck(out_channels, out_channels, 1, t, activation = activation))\n",
    "            repeat -= 1\n",
    "        \n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "def mobilenetv2(activation = 'relu'):\n",
    "    return MobileNetV2(activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27833563.99it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = mobilenetv2(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 50.0161.. Average time per step: 0.1279.. Train loss: 1.5920.. Train accuracy: 0.4065.. Top-3 train accuracy: 0.7489.. Test loss: 1.3743.. Test accuracy: 0.4929.. Top-3 test accuracy: 0.8231\n",
      "Epoch 2/100.. Time per epoch: 49.2244.. Average time per step: 0.1259.. Train loss: 1.2779.. Train accuracy: 0.5348.. Top-3 train accuracy: 0.8437.. Test loss: 1.2298.. Test accuracy: 0.5532.. Top-3 test accuracy: 0.8564\n",
      "Epoch 3/100.. Time per epoch: 49.3345.. Average time per step: 0.1262.. Train loss: 1.1243.. Train accuracy: 0.5955.. Top-3 train accuracy: 0.8744.. Test loss: 1.0197.. Test accuracy: 0.6389.. Top-3 test accuracy: 0.8940\n",
      "Epoch 4/100.. Time per epoch: 49.3509.. Average time per step: 0.1262.. Train loss: 1.0178.. Train accuracy: 0.6392.. Top-3 train accuracy: 0.8956.. Test loss: 0.9328.. Test accuracy: 0.6661.. Top-3 test accuracy: 0.9066\n",
      "Epoch 5/100.. Time per epoch: 49.3079.. Average time per step: 0.1261.. Train loss: 0.9287.. Train accuracy: 0.6720.. Top-3 train accuracy: 0.9101.. Test loss: 0.8710.. Test accuracy: 0.6887.. Top-3 test accuracy: 0.9174\n",
      "Epoch 6/100.. Time per epoch: 49.5658.. Average time per step: 0.1268.. Train loss: 0.8651.. Train accuracy: 0.6937.. Top-3 train accuracy: 0.9204.. Test loss: 0.7993.. Test accuracy: 0.7235.. Top-3 test accuracy: 0.9328\n",
      "Epoch 7/100.. Time per epoch: 49.2030.. Average time per step: 0.1258.. Train loss: 0.8079.. Train accuracy: 0.7145.. Top-3 train accuracy: 0.9281.. Test loss: 0.7428.. Test accuracy: 0.7377.. Top-3 test accuracy: 0.9381\n",
      "Epoch 8/100.. Time per epoch: 49.1281.. Average time per step: 0.1256.. Train loss: 0.7611.. Train accuracy: 0.7318.. Top-3 train accuracy: 0.9359.. Test loss: 0.6939.. Test accuracy: 0.7569.. Top-3 test accuracy: 0.9452\n",
      "Epoch 9/100.. Time per epoch: 49.1048.. Average time per step: 0.1256.. Train loss: 0.7183.. Train accuracy: 0.7497.. Top-3 train accuracy: 0.9415.. Test loss: 0.6775.. Test accuracy: 0.7662.. Top-3 test accuracy: 0.9469\n",
      "Epoch 10/100.. Time per epoch: 49.0894.. Average time per step: 0.1255.. Train loss: 0.6910.. Train accuracy: 0.7568.. Top-3 train accuracy: 0.9452.. Test loss: 0.6412.. Test accuracy: 0.7819.. Top-3 test accuracy: 0.9525\n",
      "Epoch 11/100.. Time per epoch: 49.3542.. Average time per step: 0.1262.. Train loss: 0.6621.. Train accuracy: 0.7665.. Top-3 train accuracy: 0.9484.. Test loss: 0.6534.. Test accuracy: 0.7745.. Top-3 test accuracy: 0.9519\n",
      "Epoch 12/100.. Time per epoch: 49.6525.. Average time per step: 0.1270.. Train loss: 0.6376.. Train accuracy: 0.7759.. Top-3 train accuracy: 0.9522.. Test loss: 0.6334.. Test accuracy: 0.7796.. Top-3 test accuracy: 0.9526\n",
      "Epoch 13/100.. Time per epoch: 49.4332.. Average time per step: 0.1264.. Train loss: 0.6105.. Train accuracy: 0.7876.. Top-3 train accuracy: 0.9550.. Test loss: 0.6091.. Test accuracy: 0.7952.. Top-3 test accuracy: 0.9574\n",
      "Epoch 14/100.. Time per epoch: 49.4405.. Average time per step: 0.1264.. Train loss: 0.5958.. Train accuracy: 0.7906.. Top-3 train accuracy: 0.9557.. Test loss: 0.5724.. Test accuracy: 0.7990.. Top-3 test accuracy: 0.9613\n",
      "Epoch 15/100.. Time per epoch: 49.2260.. Average time per step: 0.1259.. Train loss: 0.5723.. Train accuracy: 0.8007.. Top-3 train accuracy: 0.9601.. Test loss: 0.5628.. Test accuracy: 0.8026.. Top-3 test accuracy: 0.9601\n",
      "Epoch 16/100.. Time per epoch: 49.2943.. Average time per step: 0.1261.. Train loss: 0.5506.. Train accuracy: 0.8056.. Top-3 train accuracy: 0.9620.. Test loss: 0.5694.. Test accuracy: 0.8058.. Top-3 test accuracy: 0.9594\n",
      "Epoch 17/100.. Time per epoch: 49.3609.. Average time per step: 0.1262.. Train loss: 0.5384.. Train accuracy: 0.8123.. Top-3 train accuracy: 0.9644.. Test loss: 0.5419.. Test accuracy: 0.8201.. Top-3 test accuracy: 0.9647\n",
      "Epoch 18/100.. Time per epoch: 49.4663.. Average time per step: 0.1265.. Train loss: 0.5257.. Train accuracy: 0.8165.. Top-3 train accuracy: 0.9637.. Test loss: 0.5256.. Test accuracy: 0.8196.. Top-3 test accuracy: 0.9669\n",
      "Epoch 19/100.. Time per epoch: 49.0212.. Average time per step: 0.1254.. Train loss: 0.5096.. Train accuracy: 0.8230.. Top-3 train accuracy: 0.9670.. Test loss: 0.5576.. Test accuracy: 0.8107.. Top-3 test accuracy: 0.9611\n",
      "Epoch 20/100.. Time per epoch: 49.1361.. Average time per step: 0.1257.. Train loss: 0.4943.. Train accuracy: 0.8277.. Top-3 train accuracy: 0.9689.. Test loss: 0.5279.. Test accuracy: 0.8209.. Top-3 test accuracy: 0.9641\n",
      "Epoch 21/100.. Time per epoch: 49.3135.. Average time per step: 0.1261.. Train loss: 0.4825.. Train accuracy: 0.8315.. Top-3 train accuracy: 0.9695.. Test loss: 0.5140.. Test accuracy: 0.8279.. Top-3 test accuracy: 0.9673\n",
      "Epoch 22/100.. Time per epoch: 49.2309.. Average time per step: 0.1259.. Train loss: 0.4676.. Train accuracy: 0.8367.. Top-3 train accuracy: 0.9711.. Test loss: 0.5006.. Test accuracy: 0.8299.. Top-3 test accuracy: 0.9669\n",
      "Epoch 23/100.. Time per epoch: 49.3976.. Average time per step: 0.1263.. Train loss: 0.4591.. Train accuracy: 0.8418.. Top-3 train accuracy: 0.9721.. Test loss: 0.5151.. Test accuracy: 0.8218.. Top-3 test accuracy: 0.9688\n",
      "Epoch 24/100.. Time per epoch: 49.2944.. Average time per step: 0.1261.. Train loss: 0.4485.. Train accuracy: 0.8431.. Top-3 train accuracy: 0.9732.. Test loss: 0.4938.. Test accuracy: 0.8322.. Top-3 test accuracy: 0.9690\n",
      "Epoch 25/100.. Time per epoch: 49.2617.. Average time per step: 0.1260.. Train loss: 0.4356.. Train accuracy: 0.8483.. Top-3 train accuracy: 0.9744.. Test loss: 0.4859.. Test accuracy: 0.8344.. Top-3 test accuracy: 0.9700\n",
      "Epoch 26/100.. Time per epoch: 49.2152.. Average time per step: 0.1259.. Train loss: 0.4260.. Train accuracy: 0.8497.. Top-3 train accuracy: 0.9757.. Test loss: 0.4815.. Test accuracy: 0.8379.. Top-3 test accuracy: 0.9703\n",
      "Epoch 27/100.. Time per epoch: 49.1059.. Average time per step: 0.1256.. Train loss: 0.4179.. Train accuracy: 0.8548.. Top-3 train accuracy: 0.9760.. Test loss: 0.4963.. Test accuracy: 0.8369.. Top-3 test accuracy: 0.9693\n",
      "Epoch 28/100.. Time per epoch: 49.5384.. Average time per step: 0.1267.. Train loss: 0.4046.. Train accuracy: 0.8588.. Top-3 train accuracy: 0.9775.. Test loss: 0.4882.. Test accuracy: 0.8351.. Top-3 test accuracy: 0.9701\n",
      "Epoch 29/100.. Time per epoch: 49.2906.. Average time per step: 0.1261.. Train loss: 0.4017.. Train accuracy: 0.8603.. Top-3 train accuracy: 0.9775.. Test loss: 0.4694.. Test accuracy: 0.8444.. Top-3 test accuracy: 0.9724\n",
      "Epoch 30/100.. Time per epoch: 49.2240.. Average time per step: 0.1259.. Train loss: 0.3877.. Train accuracy: 0.8656.. Top-3 train accuracy: 0.9791.. Test loss: 0.4885.. Test accuracy: 0.8398.. Top-3 test accuracy: 0.9684\n",
      "Epoch 31/100.. Time per epoch: 49.2258.. Average time per step: 0.1259.. Train loss: 0.3802.. Train accuracy: 0.8655.. Top-3 train accuracy: 0.9803.. Test loss: 0.4780.. Test accuracy: 0.8405.. Top-3 test accuracy: 0.9701\n",
      "Epoch 32/100.. Time per epoch: 49.1094.. Average time per step: 0.1256.. Train loss: 0.3744.. Train accuracy: 0.8691.. Top-3 train accuracy: 0.9806.. Test loss: 0.4852.. Test accuracy: 0.8390.. Top-3 test accuracy: 0.9706\n",
      "Epoch 33/100.. Time per epoch: 49.0634.. Average time per step: 0.1255.. Train loss: 0.3653.. Train accuracy: 0.8701.. Top-3 train accuracy: 0.9814.. Test loss: 0.4883.. Test accuracy: 0.8400.. Top-3 test accuracy: 0.9699\n",
      "Epoch 34/100.. Time per epoch: 49.3737.. Average time per step: 0.1263.. Train loss: 0.3556.. Train accuracy: 0.8758.. Top-3 train accuracy: 0.9816.. Test loss: 0.4954.. Test accuracy: 0.8347.. Top-3 test accuracy: 0.9716\n",
      "Epoch 35/100.. Time per epoch: 49.2571.. Average time per step: 0.1260.. Train loss: 0.3484.. Train accuracy: 0.8769.. Top-3 train accuracy: 0.9822.. Test loss: 0.4797.. Test accuracy: 0.8433.. Top-3 test accuracy: 0.9719\n",
      "Epoch 36/100.. Time per epoch: 49.3089.. Average time per step: 0.1261.. Train loss: 0.3425.. Train accuracy: 0.8806.. Top-3 train accuracy: 0.9837.. Test loss: 0.4652.. Test accuracy: 0.8485.. Top-3 test accuracy: 0.9733\n",
      "Epoch 37/100.. Time per epoch: 49.0445.. Average time per step: 0.1254.. Train loss: 0.3369.. Train accuracy: 0.8823.. Top-3 train accuracy: 0.9840.. Test loss: 0.4624.. Test accuracy: 0.8524.. Top-3 test accuracy: 0.9726\n",
      "Epoch 38/100.. Time per epoch: 49.0034.. Average time per step: 0.1253.. Train loss: 0.3289.. Train accuracy: 0.8837.. Top-3 train accuracy: 0.9839.. Test loss: 0.4521.. Test accuracy: 0.8533.. Top-3 test accuracy: 0.9724\n",
      "Epoch 39/100.. Time per epoch: 49.3653.. Average time per step: 0.1263.. Train loss: 0.3238.. Train accuracy: 0.8863.. Top-3 train accuracy: 0.9850.. Test loss: 0.4555.. Test accuracy: 0.8523.. Top-3 test accuracy: 0.9733\n",
      "Epoch 40/100.. Time per epoch: 49.4488.. Average time per step: 0.1265.. Train loss: 0.3148.. Train accuracy: 0.8907.. Top-3 train accuracy: 0.9852.. Test loss: 0.4642.. Test accuracy: 0.8476.. Top-3 test accuracy: 0.9741\n",
      "Epoch 41/100.. Time per epoch: 49.5971.. Average time per step: 0.1268.. Train loss: 0.3108.. Train accuracy: 0.8901.. Top-3 train accuracy: 0.9859.. Test loss: 0.4573.. Test accuracy: 0.8500.. Top-3 test accuracy: 0.9739\n",
      "Epoch 42/100.. Time per epoch: 49.3506.. Average time per step: 0.1262.. Train loss: 0.3046.. Train accuracy: 0.8939.. Top-3 train accuracy: 0.9862.. Test loss: 0.4587.. Test accuracy: 0.8508.. Top-3 test accuracy: 0.9730\n",
      "Epoch 43/100.. Time per epoch: 49.2659.. Average time per step: 0.1260.. Train loss: 0.2954.. Train accuracy: 0.8944.. Top-3 train accuracy: 0.9869.. Test loss: 0.4702.. Test accuracy: 0.8486.. Top-3 test accuracy: 0.9735\n",
      "Epoch 44/100.. Time per epoch: 49.0665.. Average time per step: 0.1255.. Train loss: 0.2920.. Train accuracy: 0.8971.. Top-3 train accuracy: 0.9877.. Test loss: 0.4800.. Test accuracy: 0.8517.. Top-3 test accuracy: 0.9740\n",
      "Epoch 45/100.. Time per epoch: 49.2341.. Average time per step: 0.1259.. Train loss: 0.2893.. Train accuracy: 0.8985.. Top-3 train accuracy: 0.9882.. Test loss: 0.4673.. Test accuracy: 0.8521.. Top-3 test accuracy: 0.9745\n",
      "Epoch 46/100.. Time per epoch: 49.3701.. Average time per step: 0.1263.. Train loss: 0.2811.. Train accuracy: 0.9014.. Top-3 train accuracy: 0.9887.. Test loss: 0.4794.. Test accuracy: 0.8468.. Top-3 test accuracy: 0.9719\n",
      "Epoch 47/100.. Time per epoch: 49.4218.. Average time per step: 0.1264.. Train loss: 0.2744.. Train accuracy: 0.9043.. Top-3 train accuracy: 0.9888.. Test loss: 0.4668.. Test accuracy: 0.8487.. Top-3 test accuracy: 0.9722\n",
      "Epoch 48/100.. Time per epoch: 49.2330.. Average time per step: 0.1259.. Train loss: 0.2737.. Train accuracy: 0.9032.. Top-3 train accuracy: 0.9884.. Test loss: 0.4646.. Test accuracy: 0.8533.. Top-3 test accuracy: 0.9725\n",
      "Epoch 49/100.. Time per epoch: 49.2523.. Average time per step: 0.1260.. Train loss: 0.2696.. Train accuracy: 0.9056.. Top-3 train accuracy: 0.9893.. Test loss: 0.4825.. Test accuracy: 0.8507.. Top-3 test accuracy: 0.9748\n",
      "Epoch 50/100.. Time per epoch: 49.0072.. Average time per step: 0.1253.. Train loss: 0.2615.. Train accuracy: 0.9079.. Top-3 train accuracy: 0.9898.. Test loss: 0.4726.. Test accuracy: 0.8542.. Top-3 test accuracy: 0.9746\n",
      "Epoch 51/100.. Time per epoch: 49.2732.. Average time per step: 0.1260.. Train loss: 0.2578.. Train accuracy: 0.9091.. Top-3 train accuracy: 0.9900.. Test loss: 0.4755.. Test accuracy: 0.8534.. Top-3 test accuracy: 0.9724\n",
      "Epoch 52/100.. Time per epoch: 49.1854.. Average time per step: 0.1258.. Train loss: 0.2530.. Train accuracy: 0.9116.. Top-3 train accuracy: 0.9905.. Test loss: 0.4667.. Test accuracy: 0.8545.. Top-3 test accuracy: 0.9749\n",
      "Epoch 53/100.. Time per epoch: 49.2872.. Average time per step: 0.1261.. Train loss: 0.2519.. Train accuracy: 0.9120.. Top-3 train accuracy: 0.9903.. Test loss: 0.4797.. Test accuracy: 0.8547.. Top-3 test accuracy: 0.9732\n",
      "Epoch 54/100.. Time per epoch: 49.1327.. Average time per step: 0.1257.. Train loss: 0.2492.. Train accuracy: 0.9124.. Top-3 train accuracy: 0.9911.. Test loss: 0.4685.. Test accuracy: 0.8546.. Top-3 test accuracy: 0.9738\n",
      "Epoch 55/100.. Time per epoch: 48.9855.. Average time per step: 0.1253.. Train loss: 0.2432.. Train accuracy: 0.9145.. Top-3 train accuracy: 0.9914.. Test loss: 0.4660.. Test accuracy: 0.8598.. Top-3 test accuracy: 0.9759\n",
      "Epoch 56/100.. Time per epoch: 48.9015.. Average time per step: 0.1251.. Train loss: 0.2344.. Train accuracy: 0.9160.. Top-3 train accuracy: 0.9917.. Test loss: 0.4675.. Test accuracy: 0.8564.. Top-3 test accuracy: 0.9759\n",
      "Epoch 57/100.. Time per epoch: 49.2160.. Average time per step: 0.1259.. Train loss: 0.2344.. Train accuracy: 0.9173.. Top-3 train accuracy: 0.9919.. Test loss: 0.4821.. Test accuracy: 0.8556.. Top-3 test accuracy: 0.9730\n",
      "Epoch 58/100.. Time per epoch: 49.1221.. Average time per step: 0.1256.. Train loss: 0.2286.. Train accuracy: 0.9201.. Top-3 train accuracy: 0.9919.. Test loss: 0.4519.. Test accuracy: 0.8653.. Top-3 test accuracy: 0.9764\n",
      "Epoch 59/100.. Time per epoch: 49.2638.. Average time per step: 0.1260.. Train loss: 0.2234.. Train accuracy: 0.9213.. Top-3 train accuracy: 0.9921.. Test loss: 0.4627.. Test accuracy: 0.8635.. Top-3 test accuracy: 0.9754\n",
      "Epoch 60/100.. Time per epoch: 48.8993.. Average time per step: 0.1251.. Train loss: 0.2246.. Train accuracy: 0.9199.. Top-3 train accuracy: 0.9927.. Test loss: 0.4913.. Test accuracy: 0.8563.. Top-3 test accuracy: 0.9736\n",
      "Epoch 61/100.. Time per epoch: 49.0466.. Average time per step: 0.1254.. Train loss: 0.2170.. Train accuracy: 0.9225.. Top-3 train accuracy: 0.9928.. Test loss: 0.4634.. Test accuracy: 0.8634.. Top-3 test accuracy: 0.9762\n",
      "Epoch 62/100.. Time per epoch: 49.0805.. Average time per step: 0.1255.. Train loss: 0.2158.. Train accuracy: 0.9235.. Top-3 train accuracy: 0.9931.. Test loss: 0.4766.. Test accuracy: 0.8624.. Top-3 test accuracy: 0.9764\n",
      "Epoch 63/100.. Time per epoch: 49.1095.. Average time per step: 0.1256.. Train loss: 0.2076.. Train accuracy: 0.9258.. Top-3 train accuracy: 0.9931.. Test loss: 0.4809.. Test accuracy: 0.8626.. Top-3 test accuracy: 0.9763\n",
      "Epoch 64/100.. Time per epoch: 49.1975.. Average time per step: 0.1258.. Train loss: 0.2095.. Train accuracy: 0.9250.. Top-3 train accuracy: 0.9935.. Test loss: 0.4605.. Test accuracy: 0.8648.. Top-3 test accuracy: 0.9761\n",
      "Epoch 65/100.. Time per epoch: 49.2055.. Average time per step: 0.1258.. Train loss: 0.2009.. Train accuracy: 0.9298.. Top-3 train accuracy: 0.9935.. Test loss: 0.4757.. Test accuracy: 0.8612.. Top-3 test accuracy: 0.9759\n",
      "Epoch 66/100.. Time per epoch: 49.3061.. Average time per step: 0.1261.. Train loss: 0.2031.. Train accuracy: 0.9275.. Top-3 train accuracy: 0.9939.. Test loss: 0.5021.. Test accuracy: 0.8622.. Top-3 test accuracy: 0.9721\n",
      "Epoch 67/100.. Time per epoch: 49.1929.. Average time per step: 0.1258.. Train loss: 0.1967.. Train accuracy: 0.9301.. Top-3 train accuracy: 0.9942.. Test loss: 0.4944.. Test accuracy: 0.8585.. Top-3 test accuracy: 0.9732\n",
      "Epoch 68/100.. Time per epoch: 49.2994.. Average time per step: 0.1261.. Train loss: 0.1981.. Train accuracy: 0.9290.. Top-3 train accuracy: 0.9941.. Test loss: 0.4816.. Test accuracy: 0.8639.. Top-3 test accuracy: 0.9769\n",
      "Epoch 69/100.. Time per epoch: 49.2721.. Average time per step: 0.1260.. Train loss: 0.1925.. Train accuracy: 0.9317.. Top-3 train accuracy: 0.9948.. Test loss: 0.5102.. Test accuracy: 0.8556.. Top-3 test accuracy: 0.9737\n",
      "Epoch 70/100.. Time per epoch: 49.1159.. Average time per step: 0.1256.. Train loss: 0.1920.. Train accuracy: 0.9334.. Top-3 train accuracy: 0.9937.. Test loss: 0.4804.. Test accuracy: 0.8595.. Top-3 test accuracy: 0.9762\n",
      "Epoch 71/100.. Time per epoch: 49.3130.. Average time per step: 0.1261.. Train loss: 0.1874.. Train accuracy: 0.9333.. Top-3 train accuracy: 0.9943.. Test loss: 0.4881.. Test accuracy: 0.8592.. Top-3 test accuracy: 0.9761\n",
      "Epoch 72/100.. Time per epoch: 49.0762.. Average time per step: 0.1255.. Train loss: 0.1883.. Train accuracy: 0.9341.. Top-3 train accuracy: 0.9942.. Test loss: 0.4722.. Test accuracy: 0.8618.. Top-3 test accuracy: 0.9770\n",
      "Epoch 73/100.. Time per epoch: 49.3456.. Average time per step: 0.1262.. Train loss: 0.1814.. Train accuracy: 0.9354.. Top-3 train accuracy: 0.9949.. Test loss: 0.4716.. Test accuracy: 0.8668.. Top-3 test accuracy: 0.9759\n",
      "Epoch 74/100.. Time per epoch: 49.4066.. Average time per step: 0.1264.. Train loss: 0.1815.. Train accuracy: 0.9359.. Top-3 train accuracy: 0.9953.. Test loss: 0.5011.. Test accuracy: 0.8616.. Top-3 test accuracy: 0.9749\n",
      "Epoch 75/100.. Time per epoch: 49.3672.. Average time per step: 0.1263.. Train loss: 0.1766.. Train accuracy: 0.9381.. Top-3 train accuracy: 0.9953.. Test loss: 0.5069.. Test accuracy: 0.8585.. Top-3 test accuracy: 0.9752\n",
      "Epoch 76/100.. Time per epoch: 49.4916.. Average time per step: 0.1266.. Train loss: 0.1767.. Train accuracy: 0.9371.. Top-3 train accuracy: 0.9957.. Test loss: 0.4892.. Test accuracy: 0.8614.. Top-3 test accuracy: 0.9773\n",
      "Epoch 77/100.. Time per epoch: 49.1243.. Average time per step: 0.1256.. Train loss: 0.1735.. Train accuracy: 0.9392.. Top-3 train accuracy: 0.9951.. Test loss: 0.5152.. Test accuracy: 0.8559.. Top-3 test accuracy: 0.9761\n",
      "Epoch 78/100.. Time per epoch: 49.3876.. Average time per step: 0.1263.. Train loss: 0.1704.. Train accuracy: 0.9399.. Top-3 train accuracy: 0.9955.. Test loss: 0.4993.. Test accuracy: 0.8622.. Top-3 test accuracy: 0.9746\n",
      "Epoch 79/100.. Time per epoch: 49.2687.. Average time per step: 0.1260.. Train loss: 0.1679.. Train accuracy: 0.9411.. Top-3 train accuracy: 0.9957.. Test loss: 0.4964.. Test accuracy: 0.8636.. Top-3 test accuracy: 0.9743\n",
      "Epoch 80/100.. Time per epoch: 49.1783.. Average time per step: 0.1258.. Train loss: 0.1648.. Train accuracy: 0.9410.. Top-3 train accuracy: 0.9957.. Test loss: 0.5189.. Test accuracy: 0.8598.. Top-3 test accuracy: 0.9739\n",
      "Epoch 81/100.. Time per epoch: 49.2536.. Average time per step: 0.1260.. Train loss: 0.1582.. Train accuracy: 0.9431.. Top-3 train accuracy: 0.9962.. Test loss: 0.4824.. Test accuracy: 0.8632.. Top-3 test accuracy: 0.9779\n",
      "Epoch 82/100.. Time per epoch: 49.3509.. Average time per step: 0.1262.. Train loss: 0.1592.. Train accuracy: 0.9435.. Top-3 train accuracy: 0.9957.. Test loss: 0.4956.. Test accuracy: 0.8652.. Top-3 test accuracy: 0.9741\n",
      "Epoch 83/100.. Time per epoch: 48.9506.. Average time per step: 0.1252.. Train loss: 0.1567.. Train accuracy: 0.9443.. Top-3 train accuracy: 0.9962.. Test loss: 0.5215.. Test accuracy: 0.8560.. Top-3 test accuracy: 0.9754\n",
      "Epoch 84/100.. Time per epoch: 49.0009.. Average time per step: 0.1253.. Train loss: 0.1579.. Train accuracy: 0.9440.. Top-3 train accuracy: 0.9959.. Test loss: 0.5075.. Test accuracy: 0.8592.. Top-3 test accuracy: 0.9755\n",
      "Epoch 85/100.. Time per epoch: 49.1784.. Average time per step: 0.1258.. Train loss: 0.1515.. Train accuracy: 0.9459.. Top-3 train accuracy: 0.9965.. Test loss: 0.5070.. Test accuracy: 0.8620.. Top-3 test accuracy: 0.9773\n",
      "Epoch 86/100.. Time per epoch: 49.3025.. Average time per step: 0.1261.. Train loss: 0.1513.. Train accuracy: 0.9477.. Top-3 train accuracy: 0.9964.. Test loss: 0.5109.. Test accuracy: 0.8577.. Top-3 test accuracy: 0.9763\n",
      "Epoch 87/100.. Time per epoch: 49.1628.. Average time per step: 0.1257.. Train loss: 0.1524.. Train accuracy: 0.9469.. Top-3 train accuracy: 0.9962.. Test loss: 0.5135.. Test accuracy: 0.8611.. Top-3 test accuracy: 0.9744\n",
      "Epoch 88/100.. Time per epoch: 49.4479.. Average time per step: 0.1265.. Train loss: 0.1485.. Train accuracy: 0.9467.. Top-3 train accuracy: 0.9966.. Test loss: 0.5121.. Test accuracy: 0.8648.. Top-3 test accuracy: 0.9749\n",
      "Epoch 89/100.. Time per epoch: 49.0270.. Average time per step: 0.1254.. Train loss: 0.1478.. Train accuracy: 0.9472.. Top-3 train accuracy: 0.9962.. Test loss: 0.5335.. Test accuracy: 0.8605.. Top-3 test accuracy: 0.9733\n",
      "Epoch 90/100.. Time per epoch: 49.2707.. Average time per step: 0.1260.. Train loss: 0.1452.. Train accuracy: 0.9492.. Top-3 train accuracy: 0.9962.. Test loss: 0.5122.. Test accuracy: 0.8633.. Top-3 test accuracy: 0.9764\n",
      "Epoch 91/100.. Time per epoch: 49.2198.. Average time per step: 0.1259.. Train loss: 0.1397.. Train accuracy: 0.9511.. Top-3 train accuracy: 0.9968.. Test loss: 0.5469.. Test accuracy: 0.8596.. Top-3 test accuracy: 0.9759\n",
      "Epoch 92/100.. Time per epoch: 49.0830.. Average time per step: 0.1255.. Train loss: 0.1456.. Train accuracy: 0.9486.. Top-3 train accuracy: 0.9966.. Test loss: 0.5136.. Test accuracy: 0.8647.. Top-3 test accuracy: 0.9759\n",
      "Epoch 93/100.. Time per epoch: 48.9928.. Average time per step: 0.1253.. Train loss: 0.1402.. Train accuracy: 0.9514.. Top-3 train accuracy: 0.9972.. Test loss: 0.5181.. Test accuracy: 0.8659.. Top-3 test accuracy: 0.9758\n",
      "Epoch 94/100.. Time per epoch: 49.5230.. Average time per step: 0.1267.. Train loss: 0.1393.. Train accuracy: 0.9507.. Top-3 train accuracy: 0.9973.. Test loss: 0.5156.. Test accuracy: 0.8651.. Top-3 test accuracy: 0.9759\n",
      "Epoch 95/100.. Time per epoch: 49.1544.. Average time per step: 0.1257.. Train loss: 0.1375.. Train accuracy: 0.9512.. Top-3 train accuracy: 0.9970.. Test loss: 0.5396.. Test accuracy: 0.8613.. Top-3 test accuracy: 0.9750\n",
      "Epoch 96/100.. Time per epoch: 49.2886.. Average time per step: 0.1261.. Train loss: 0.1366.. Train accuracy: 0.9511.. Top-3 train accuracy: 0.9970.. Test loss: 0.5294.. Test accuracy: 0.8616.. Top-3 test accuracy: 0.9750\n",
      "Epoch 97/100.. Time per epoch: 49.4162.. Average time per step: 0.1264.. Train loss: 0.1316.. Train accuracy: 0.9533.. Top-3 train accuracy: 0.9974.. Test loss: 0.5409.. Test accuracy: 0.8620.. Top-3 test accuracy: 0.9729\n",
      "Epoch 98/100.. Time per epoch: 49.0621.. Average time per step: 0.1255.. Train loss: 0.1355.. Train accuracy: 0.9526.. Top-3 train accuracy: 0.9970.. Test loss: 0.5337.. Test accuracy: 0.8595.. Top-3 test accuracy: 0.9747\n",
      "Epoch 99/100.. Time per epoch: 49.0411.. Average time per step: 0.1254.. Train loss: 0.1289.. Train accuracy: 0.9544.. Top-3 train accuracy: 0.9975.. Test loss: 0.5219.. Test accuracy: 0.8679.. Top-3 test accuracy: 0.9762\n",
      "Epoch 100/100.. Time per epoch: 49.2957.. Average time per step: 0.1261.. Train loss: 0.1273.. Train accuracy: 0.9543.. Top-3 train accuracy: 0.9973.. Test loss: 0.5269.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9754\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_MobileNetv2_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
